import sys
import time
from ast import literal_eval

import redis

from setting import Setting
import csdn_spider
from hashlib import md5
from threading import Thread
import urllib.request  as ur
import  urllib.parse as up
from queue import Queue
#核心方法,
class Core():
    def __init__(self,spider_filename,spider,setting,middleware_s,pipeline_s):
        self.spider_filename = spider_filename
        self.spider = spider
        self.setting = setting
        self.middleware_s = middleware_s
        self.pipeline_s = pipeline_s

        self.redis_conn = redis.StrictRedis(**setting.REDIS_CONN_CONFIG)
        #已经发送过请求的集合
        self.requested_key_name = '%s:requested'% spider_filename
        #调度器集合
        self.scheduler_key_name = '%s:scheduler_request'% spider_filename
        # 请求队列
        self.request_queue = Queue()
    # 把request的请求先交给调度器
    def scheduler(self):
        def put_request(request):
            # 验证地址是否发送过
            requested_old_num = self.redis_conn.scard(self.requested_key_name)
            request_md5 = md5(
                (request.url + str(request.data)).encode('utf-8')
            ).hexdigest()
            self.redis_conn.sadd(self.requested_key_name, request_md5)
            requested_new_num = self.redis_conn.scard(self.requested_key_name)
            if requested_new_num > requested_old_num:
                request_dict = dict(
                    url=request.url,
                    data=request.data,
                    header=request.headers,
                    callback=request.callback,
                )
                self.redis_conn.sadd(
                    self.scheduler_key_name,
                    str(request_dict)
                )
        for request in self.spider.start_request():
            put_request(request)

        #如果没有发送过交给调度器
        while True:
            request = self.request_queue.get()
            put_request(request)
    # 调度器和处理器是应该并行操作
    def process(self):
        while True:
            request_str = self.redis_conn.spop(self.scheduler_key_name)
            if not request_str:
                time.sleep(1)
                continue
            # 把字符串变成字典
            request_dict = literal_eval(request_str)
            request = ur.Request(
                url=request_dict['url'],
                data=request_dict['data'],
                headers=request_dict['header']
            )
            # todo 中间件执行before_request
            response = ur.urlopen(request)
            #todo 中间
            #回调函数
            for data in eval('self.spider.%s(response)'% request_dict['callback']):
                #测试时把Redis清空,不然以为已经过滤掉,得不到信息
                if isinstance(data,csdn_spider.Request):
                    # 交给调度器
                    self.request_queue.put(data)
                else:
                    # 交给pipeline
                    for pipeline in pipeline_s:
                        pipeline.process_item(data)



    def run(self):
        # 这两个函数就以多线程来运行
        thread_scheduler = Thread(target=self.scheduler)
        thread_scheduler.start()
        thread_process = Thread(target=self.process)
        thread_process.start()

if __name__ == '__main__':
    #python manage.py s1   通过sys.argv 拿到s1
    spider_filename = sys.argv[1]
    try:
        #exec 把str转成可以运行的代码  和eval的区别:exec没有返回值,eval有返回值,危险,任何代码都能执行;literal_eval有些命令不能执行,安全
        #s1.Spider类动态导入到本文件
        # from spider.s1 import Spider
        exec('from spider.%s import Spider' % spider_filename)
        # 实例化Spider类
        spider:csdn_spider.Spider= Spider()
        print(spider)
    except:
        raise (Exception('spider file not found or correct'))

    #实例化Setting
    setting = Setting()
    #创建列表,接受实例化的中间件
    middleware_s = []
    #对中间件遍历
    for middleware in setting.MIDDLEWARES:
        filepath,classname = middleware.rsplit('.',maxsplit=1)
        # print(filepath,classname)
        exec('from %s import %s' % (filepath,classname))
        middleware_s.append(eval('%s()' % classname))
    print(middleware_s)
    pipeline_s = []
    for pipeline in setting.PIPELINES:
        filepath, classname = pipeline.rsplit('.', maxsplit=1)
        exec('from %s import %s'%(filepath,classname))
        pipeline_s.append(eval('%s()' % classname))

    #在这里搞中间件有什么用呢?
    #让请求经过中间件处理再返回,拿到处理过的request
    core = Core(spider_filename,spider,setting,middleware_s,pipeline_s)
    core.run()